
ulimit -u unlimited
print_v(){
    # @arg1: name
    echo $1=${!1}
}
get_value_from_env(){
    # @arg1: name
    # @arg2: default value
    if [ "${!1}" == '' ]
    then
        export $1=$2
    fi
}
get_real_path(){
  if [ "${1:0:1}" == "/" ]; then
    echo "$1"
  else
    echo "$(realpath -m $PWD/$1)"
  fi
}
kill_profile(){
    ps -ef | grep profile_alone | grep -v grep |  cut -c 9-15 | xargs kill -9 # kill profile thread if exists
    ps -ef | grep testADSLab | grep -v grep |  cut -c 9-15 | xargs kill -9
    ps -ef | grep python | grep -v grep |  cut -c 9-15 | xargs kill -9 
}


CURRENT_DIRECTORY=$(pwd)
get_value_from_env PROC_TITLE testADSLab
get_value_from_env IMAGENET "/data/DNN_Dataset/imagenet/tiny/meshtf/mininet/mininet/mini-imagenet-sp2/val/"
get_value_from_env WIKI "/data/DNN_Dataset/zhwiki/tfdir/"
get_value_from_env LAST_SEC 1800
get_value_from_env RESULT_BASE_DIR "/data/result/mtf/"
OVER_FLAG="over"

# RESNET_DIR="${CURRENT_DIRECTORY}/../.."
# VGG_DIR="${CURRENT_DIRECTORY}/../.."
CV_DIR="${CURRENT_DIRECTORY}/../.."
BERT_DIR="${CURRENT_DIRECTORY}/../bert"



print_v CV_DIR
print_v BERT_DIR


profile_cv(){
    # always auto parallel
    kill_profile
    export MODEL="$1"
    export DEVICE_NUM="$2"
    export PARALLEL_MODE="$3"
    export BATCHSIZE="$4"
    export ID="${MODEL}_${DEVICE_NUM}_${PARALLEL_MODE}"
    export STORE_DIR="${RESULT_BASE_DIR}/${ID}"  # 子文件夹路径
    export RUNNING_DIR=$CV_DIR
    export DATASET=imagenet2012
    export DATA_PATH=$IMAGENET
    mkdir -p $STORE_DIR
    echo ID=$ID
    if [ ! -f "$STORE_DIR/$OVER_FLAG" ]
    then
        cd $RUNNING_DIR
        python adsl4mtf/run.py \
            --data_url="${DATA_PATH}" \
            --num_gpus=${DEVICE_NUM} \
            --models=${MODEL} \
            --class_nums=1000 \
            &> ${STORE_DIR}/log.txt &
        cd $CURRENT_DIRECTORY
        bash ./profile_alone.sh &> ${STORE_DIR}/monitor.log &
        echo "Wait $LAST_SEC seconds to kill..." 
        sleep $LAST_SEC
        pkill $PROC_TITLE
        touch $STORE_DIR/$OVER_FLAG
    fi
    echo Experiment [$ID] finished
}

profile_bert(){
    # arg1: model [base, large] 
    # arg2: device_num: 总共的卡数
    # arg3: parallel_mode: [data_parallel, auto_parallel]
    # arg4: batchsize
    # arg5: optimizer: [adam, Lamb, Momentum] only adam
    kill_profile
    export BERT_NETWORK="$1"
    export MODEL="BERT-$1"
    export DEVICE_NUM="$2"
    export PARALLEL_MODE="$3"
    export BATCHSIZE="$4"
    export OPTIMIZER="$5"
    export ID="${MODEL}_${DEVICE_NUM}_${PARALLEL_MODE}"
    export STORE_DIR="${RESULT_BASE_DIR}/${ID}"  # 子文件夹路径
    export RUNNING_DIR=$BERT_DIR
    export DATASET=wiki
    export DATA_PATH=$WIKI
    mkdir -p $STORE_DIR
    echo ID=$ID
    if [ ! -f "$STORE_DIR/$OVER_FLAG" ]
    then
        cd $RUNNING_DIR
        python run_pretraining.py \
            --mode=$PARALLEL_MODE \
            --bert_config_file=./${BERT_NETWORK}.json \
            --input_train_files=$DATA_PATH \
            --output_dir=$STORE_DIR/output \
            --gpu_num=$DEVICE_NUM \
            --train_batch_size=$BATCHSIZE \
            &> ${STORE_DIR}/log.txt &
        cd $CURRENT_DIRECTORY
        bash ./profile_alone.sh &> ${STORE_DIR}/monitor.log &
        echo "Wait $LAST_SEC seconds to kill..." 
        sleep $LAST_SEC
        pkill $PROC_TITLE
        touch $STORE_DIR/$OVER_FLAG
    fi
    echo Experiment [$ID] finished
}

profile_cv resnet50 8 auto_parallel 32
profile_cv resnet50 4 auto_parallel 32
profile_cv resnet50 2 auto_parallel 32
profile_cv resnet50 1 auto_parallel 32
profile_cv vgg16 8 auto_parallel 32
profile_cv vgg16 4 auto_parallel 32
profile_cv vgg16 2 auto_parallel 32
profile_cv vgg16 1 auto_parallel 32

profile_cv resnet50 8 auto_parallel 64
profile_cv resnet50 4 auto_parallel 64
profile_cv resnet50 2 auto_parallel 64
profile_cv resnet50 1 auto_parallel 64
profile_cv vgg16 8 auto_parallel 64
profile_cv vgg16 4 auto_parallel 64
profile_cv vgg16 2 auto_parallel 64
profile_cv vgg16 1 auto_parallel 64

exit

profile_bert base 8 auto_parallel 32 adam
profile_bert large 8 auto_parallel 32 adam
profile_bert base 8 auto_parallel 64 adam
profile_bert large 8 auto_parallel 64 adam
profile_cv vgg16 8 auto_parallel 32
profile_cv vgg19 8 auto_parallel 32
profile_cv vgg16 8 auto_parallel 64
profile_cv vgg19 8 auto_parallel 64
profile_cv resnet50 8 auto_parallel 32
profile_cv resnet101 8 auto_parallel 32
profile_cv resnet50 8 auto_parallel 64
profile_cv resnet101 8 auto_parallel 64


profile_bert base 4 auto_parallel 32 adam
profile_bert large 4 auto_parallel 32 adam
profile_bert base 4 auto_parallel 64 adam
profile_bert large 4 auto_parallel 64 adam
profile_cv vgg16 4 auto_parallel 32
profile_cv vgg19 4 auto_parallel 32
profile_cv vgg16 4 auto_parallel 64
profile_cv vgg19 4 auto_parallel 64
profile_cv resnet50 4 auto_parallel 32
profile_cv resnet101 4 auto_parallel 32
profile_cv resnet50 4 auto_parallel 64
profile_cv resnet101 4 auto_parallel 64


profile_bert base 2 auto_parallel 32 adam
profile_bert large 2 auto_parallel 32 adam
profile_bert base 2 auto_parallel 64 adam
profile_bert large 2 auto_parallel 64 adam
profile_cv vgg16 2 auto_parallel 32
profile_cv vgg19 2 auto_parallel 32
profile_cv vgg16 2 auto_parallel 64
profile_cv vgg19 2 auto_parallel 64
profile_cv resnet50 2 auto_parallel 32
profile_cv resnet101 2 auto_parallel 32
profile_cv resnet50 2 auto_parallel 64
profile_cv resnet101 2 auto_parallel 64


profile_bert base 1 auto_parallel 32 adam
profile_bert large 1 auto_parallel 32 adam
profile_bert base 1 auto_parallel 64 adam
profile_bert large 1 auto_parallel 64 adam
profile_cv vgg16 1 auto_parallel 32
profile_cv vgg19 1 auto_parallel 32
profile_cv vgg16 1 auto_parallel 64
profile_cv vgg19 1 auto_parallel 64
profile_cv resnet50 1 auto_parallel 32
profile_cv resnet101 1 auto_parallel 32
profile_cv resnet50 1 auto_parallel 64
profile_cv resnet101 1 auto_parallel 64

